#!/usr/bin/env python 
# -*- coding: utf-8 -*- 
'''
Created on 2016年5月18日
@author: zhaoxiang
给产品打目的地标签，用于mysql业务部分
根据ugc_type,[内容(id)列表] 对city, country, poi进行计数统计
'''

import sys
import json,types,copy
import os
sys.path.append('/home/hadoop/udf_data/python/user_track')
os.environ['PYTHON_EGG_CACHE'] = '/tmp/hadoop/.python-eggs/'
os.environ['PYTHON_EGG_DIR']='/tmp/hadoop/.python-eggs/'
from DB import *
import gua
import jieba
from util_func import dateToTimestamp, sql_transform, strdecode
from sql_util import horizonal_sql


#将产品内容打上目的地（国家/城市）标签
class ProductTagger(object) :
    def __init__(self) :
        self.qyer_db = DB(gua.conn_qyer['host'], gua.conn_qyer['username'], gua.conn_qyer['password'], gua.conn_qyer['db'], gua.conn_qyer["port"])
        self.ra_db = DB(gua.conn_ra['host'], gua.conn_ra['username'], gua.conn_ra['password'], gua.conn_ra['db'], gua.conn_ra["port"])
        self.plan_db = DB(gua.conn_plan["host"], gua.conn_plan["username"], gua.conn_plan["password"], gua.conn_plan["db"], gua.conn_plan["port"])
        self.main_db = DB(gua.conn_main["host"], gua.conn_main["username"], gua.conn_main["password"], gua.conn_main["db"])
        self.taiwan_set = self.get_taiwan()
        self.city_urlname = {}
        self.country_urlname = {}
        self.city_country = self.init_city_country()
        self.poi_topic = self.init_poi_topic()
        self.guide_urlname = {}
        self.lm_ptype_urlname = {}
        self.lm_dept_urlname = {}
        self.lm_travel_urlname = {}
#         jieba.load_userdict(DICT_FILE)

    def __del__(self) :
        self.qyer_db.close()
        self.plan_db.close()
        self.ra_db.close()

    #########################################################
    #功能:初始化折扣url相关关键词                           #
    #########################################################
    def init_lm_urlname(self) :
        self.lm_ptype_urlname = {"flights":1016, "hotels":1017, "cruise":1020, "car":1021, "safe":1049, "qz":1785, "local":2410, "ot":2161, "other":1050, "free":1018, "freetour":1018} 
        self.lm_dept_urlname = {"1":"bjtj", "2":"shhz", "3":"cdcq", "4":"gzsz", "5":"hmt", "6":"inland", "7":"abroad"}
        self.lm_travel_urlname = {"1":"1", "2":"1-3", "3":"3-6", "4":"6", "5":"元旦", "6":"春节", "7":"清明", "8":"劳动", "9":"端午", "10":"中秋", "11":"国庆"}

    #########################################################
    #功能:初始化城市对应的国家                              #
    #输出:字典 {city_id:country_id}                         #
    #########################################################
    def init_city_country(self) :
        city_country = {}
        self.qyer_db.execute("select id, countryid from city")
        result_set = self.qyer_db.fetchall()
        for result in result_set :
            city_id = result["id"]
            country_id = result["countryid"]
            if city_id == 50 or city_id == 51 : country_id = -city_id
            elif city_id in self.taiwan_set : country_id = -11186
            elif country_id == 11 : continue
            city_country[city_id] = country_id
        return city_country
    
    def init_poi_topic(self):
        poi_topic = {}
        self.qyer_db.execute("select id,tagid,cateid from place_poi where status = 1 and overall_rank < 9999999 and hotrank > 0 and length(tagid) > 0")
        result_set = self.qyer_db.fetchall()
        for result in result_set:
            poi_id = int(result['id'])
            tag_id = set([ int(tag) for tag in result['tagid'].strip().split(',') ])
            cateid = int(result['cateid'])
            poi_topic[poi_id] = (list(tag_id),cateid)
        return poi_topic

    #####################################################
    #功能:根据城市英文名，初始化城市url                 #
    #输出:字典 {城市url名:城市id}                       #
    #####################################################
    def translate_city_urlname(self) :
        self.qyer_db.execute("select id, cityname, cityname_en, cityname_url from city where status = 1")
        result_set = self.qyer_db.fetchall()
        for result in result_set :
            #cnname = result['cityname'].strip("\t ")
            cityid = result['id']
            enname = result['cityname_en'].lower().strip("\t ")
            cityname_url = result['cityname_url']
            if cityname_url : enname = cityname_url.lower().strip("\t ")
            urlname = enname.replace("'", "").replace(" ", "-").replace("&", "and").replace(",", "-").replace(".", "")
            self.city_urlname[urlname] = cityid #cnname

    #####################################################
    #功能:根据国家英文名，初始化国家url                 #
    #输出:字典 {国家url名:国家id}                       #
    #####################################################
    def translate_country_urlname(self) :
        self.qyer_db.execute("select id, catename, catename_en from category where parentid in (select id from category where parentid = 1)")
        result_set = self.qyer_db.fetchall()
        for result in result_set :
            countryid = result['id']
            #cnname = result['catename']
            enname = result['catename_en'].lower()
            urlname = enname.replace("'", "").replace(" ", "-").replace("&", "and").replace(",", "-").replace(".", "")
            self.country_urlname[urlname] = countryid

    #####################################################
    #功能:根据锦囊英文名，初始化锦囊url                 #
    #输出:字典 {锦囊url名:锦囊id}                       #
    #####################################################
    def translate_guide_urlname(self) :
        self.qyer_db.execute("select id, enname from qyer.jn_info where status = 1")
        result_set = self.qyer_db.fetchall()
        for result in result_set :
            guide_id = result['id']
            enname = result['enname'].lower()
            urlname = enname.replace("'", "").replace(" ", "-").replace("&", "and").replace(",", "-").replace(".", "")
            self.guide_urlname[urlname] = guide_id

    #####################################################
    #功能:获取台湾省城市集合                            #
    #输出:城市id集合                                    #
    #####################################################
    def get_taiwan(self) :
        self.qyer_db.execute("select id from city where countryid = 11 and id not in (50,51) and status = 1")
        result_set = self.qyer_db.fetchall()
        return set([result["id"] for result in result_set])

    #################################################################
    #功能:给产品打目的地标签                                        #
    #输入:type - string, 行为类型                                   #
    #     tagged_list - 与行为对应的相应产品的id列表                #
    #输出:字典                                                      #
    #     {"country":{country_id:country_id在产品列表中出现的次数}, #
    #      "city":{city_id:city_id在产品列表中出现的次数}}          #
    #################################################################
    def product_tag(self, type, tagged_list) :
        res_tagged_list = {}
        if type == 'v_tbbs' or type == "p_travel" or type == "coplay" : res_tagged_list = self.thread_tag(tagged_list)
        elif type == 'v_ask' or type == 'ask' or type == 'answer' : res_tagged_list = self.question_tag(tagged_list)
        elif type == 'v_plan' or type == 'plan' or type == "edit_plan" : res_tagged_list = self.plan_tag(tagged_list)
        elif type == 'search' : res_tagged_list = self.search_tag(tagged_list)
        elif type == 'v_place' : res_tagged_list = self.place_tag(tagged_list)
        elif type == 'v_guide' or type == 'd_guide' or type == 'd_web_guide' : res_tagged_list = self.guide_tag(tagged_list)
        elif type == 'v_lm' or type == 'buy_lm' or type == "collect_lm" : res_tagged_list = self.lm_tag(tagged_list)
        elif type == "v_lmlist" : res_tagged_list = self.lmlist_tag(tagged_list)
        elif type == 'p_comment' : res_tagged_list = self.pcomment_tag(tagged_list)
        elif type == 'v_mguide' : res_tagged_list = self.mguide_tag(tagged_list)
        elif type == "v_hotel" or type == "book_hotel" or type == "v_goto_hotel" : res_tagged_list = self.hotel_tag(tagged_list)
        elif type == "v_hotellist" or type == "v_glist" : res_tagged_list = self.xlist_tag(tagged_list)
        return res_tagged_list

    #################################################################
    #功能:给论坛打目的地标签                                        #
    #输入:tid_list - tid列表                                        #
    #输出:字典                                                      #
    #     {"country":{country_id:country_id在产品列表中出现的次数}, #
    #      "city":{city_id:city_id在产品列表中出现的次数}}          #
    #################################################################
    def thread_tag(self, tid_list) :
        country_dict = {}
        city_dict = {}
        topic_dict = {}
        cateid_dict = {}
        poiid_dict = {}
        for tid in tid_list :
            thread_country_list = []
            thread_city_list = []
            self.ra_db.execute("select country_id, city_ids, poi_ids from tag.bbs_thread_tag_place where tid = %s and status = 1", (tid,))
            tag_result_set = self.ra_db.fetchall()
            for tag_result in tag_result_set :
                country_id = tag_result["country_id"]
                topic_ids = []
                cate_ids = []
                try :       city_ids = map(int, tag_result["city_ids"].split(","))
                except : city_ids = []
                try : poi_ids = map(int, tag_result["poi_ids"].split(","))
                except : poi_ids = []
                for poi in poi_ids:
                    poiid_dict[poi] = poiid_dict.get(poi,0) + 1
                    if poi in self.poi_topic : 
                        for topic in self.poi_topic[poi][0]: topic_ids.append(topic)
                        cate_ids.append(self.poi_topic[poi][1])
                country_dict[country_id] = country_dict.get(country_id, 0) + 1
                
                for cate_id in cate_ids: cateid_dict[cate_id] = cateid_dict.get(cate_id,0) + 1
                for topic_id in topic_ids: topic_dict[topic_id] = topic_dict.get(topic_id,0) + 1
                for city_id in city_ids : city_dict[city_id] = city_dict.get(city_id, 0) + 1
        return {"country":country_dict, "city":city_dict, "topic_id": topic_dict, "cate_id": cateid_dict, "poi_id":poiid_dict}

    #################################################################
    #功能:给问答打目的地标签                                        #
    #输入:qid_list - qid列表                                        #
    #输出:字典                                                      #
    #     {"country":{country_id:country_id在产品列表中出现的次数}, #
    #      "city":{city_id:city_id在产品列表中出现的次数}}          #
    #################################################################
    def question_tag(self, qid_list) :
        if len(self.city_country) == 0 : self.init_city_country()
        country_dict = {}
        city_dict = {}
        poi_dict = {}
        cateid_dict = {}
        topic_dict = {}
        for qid in qid_list :
            self.main_db.execute("select cityKeywords, countryKeywords, poiKeywords, cityKeywordsInTitle, countryKeywordsInTitle, poiKeywordsInTitle from tag.ask_tag where qid = %s", (qid,))
            result = self.main_db.fetchone()
            if not result : continue
            try : country_ids = map(int, result["countryKeywords"].split(","))
            except : country_ids = []
            try : city_ids = map(int, result["cityKeywords"].split(","))
            except : city_ids = []
            try : poi_ids = map(int, result["poiKeywords"].split(","))
            except : poi_ids = []
 
            try : city_title_ids = map(int, result["cityKeywordsInTitle"].split(","))
            except : city_title_ids = []
            city_ids += city_title_ids
            tag_cityids = list(set([city_id for city_id in city_ids if city_id > 0]))
            for city_id in tag_cityids : city_dict[city_id] = city_dict.get(city_id, 0) + 1
            
            try : country_title_ids = map(int, result["countryKeywordsInTitle"].split(","))
            except : country_title_ids = []
            country_ids += country_title_ids
            for city_id in tag_cityids : 
                if city_id in self.city_country : country_ids.append(self.city_country[city_id])
            tag_countryids = list(set(country_ids))
            
            try : poi_title_ids = map(int, result["poiKeywordsInTitle"].split(","))
            except : poi_title_ids = []
            poi_ids += poi_title_ids
            tag_poiids = set([])
            topic_ids = []
            cate_ids = []
            for poiid in poi_ids:
                if poiid > 0: tag_poiids.add(poiid)
                if poiid in self.poi_topic : 
                    for topic in self.poi_topic[poiid][0]: topic_ids.append(topic)
                    cate_ids.append(self.poi_topic[poiid][1])
            for poi_id in tag_poiids: poi_dict[poi_id] = poi_dict.get(poi_id, 0) + 1
            for cate_id in cate_ids: cateid_dict[cate_id] = cateid_dict.get(cate_id,0) + 1
            for topic_id in topic_ids: topic_dict[topic_id] = topic_dict.get(topic_id,0) + 1
            """
            tag_countryids = []
            for country_id in country_ids : tag_countryids.append(country_id) 
            #    if country_id > 0 : tag_countryids.append(country_id)
            #    else : tag_countryids.append(11)
            tag_countryids = list(set(tag_countryids))
            """
            for country_id in tag_countryids : country_dict[country_id] = country_dict.get(country_id, 0) + 1
        return {"country":country_dict, "city":city_dict, "poi":poi_dict , "topic_id": topic_dict, "cate_id": cateid_dict}


    #################################################################
    #功能:给点评打目的地标签                                        #
    #输入:comment_list - 点评列表                                   #
    #输出:字典                                                      #
    #     {"country":{country_id:country_id在产品列表中出现的次数}, #
    #      "city":{city_id:city_id在产品列表中出现的次数}}          #
    #################################################################
    def pcomment_tag(self, comment_list) :
        if len(self.city_country) == 0 : self.init_city_country()
        country_dict = {}
        for countryid in comment_list["country"] : country_dict[countryid] = country_dict.get(countryid, 0) + 1
        poi_dict = {}
        cateid_dict = {}
        topic_dict = {}
        city_dict = {}
        for cityid in comment_list["city"] :
            city_dict[cityid] = city_dict.get(cityid, 0) + 1
            if cityid in self.city_country :
                countryid = self.city_country[cityid]
                country_dict[countryid] = country_dict.get(countryid, 0) + 1
        for poiid in comment_list["poi"] :
            self.qyer_db.execute("select cityid, national from place_poi where id = %s", (poiid,))
            cate_ids = []
            topic_ids = []
            if poiid in self.poi_topic : 
                for topic in self.poi_topic[poiid][0]: topic_ids.append(topic)
                cate_ids.append(self.poi_topic[poiid][1])
            poi_dict[poiid] = poi_dict.get(poiid, 0) + 1
            for cate_id in cate_ids: cateid_dict[cate_id] = cateid_dict.get(cate_id,0) + 1
            for topic_id in topic_ids: topic_dict[topic_id] = topic_dict.get(topic_id,0) + 1
            
            result = self.qyer_db.fetchone()
            if result :
                cityid = result['cityid']
                city_dict[cityid] = city_dict.get(cityid, 0) + 1
                if cityid not in self.city_country : continue
                countryid = self.city_country[cityid]
                country_dict[countryid] = country_dict.get(countryid, 0) + 1
        return {"country":country_dict, "city":city_dict,'poi':poi_dict, "topic_id": topic_dict, "cate_id": cateid_dict}

    #################################################################
    #功能:给目的地打目的地标签                                      #
    #输入:place_detail - 字典 {"poi":[poi列表], "city":[city列表],  #
    #                          "country":[country列表]}             #
    #输出:字典                                                      #
    #     {"country":{country_id:country_id在产品列表中出现的次数}, #
    #      "city":{city_id:city_id在产品列表中出现的次数}}          #
    #################################################################
    def place_tag(self, place_detail) :
        if len(self.city_country) == 0 : self.init_city_country()
        country_dict = {}
        city_dict = {}
        poi_dict = {}
        cateid_dict = {}
        topic_dict = {}
        poi_detail = place_detail["poi"]
        if len(poi_detail) :
            self.qyer_db.execute("select id,cityid, national from place_poi where id in (%s)" % ",".join(map(str, poi_detail)))
            result_set = self.qyer_db.fetchall()
            topic_ids = [] 
            cate_ids = []
            poi_ids = []
            for result in result_set :
                cityid = int(result["cityid"])
                poiid = int(result['id'])
                if cityid not in self.city_country : continue
                countryid = int(self.city_country[cityid])
                country_dict[countryid] = country_dict.get(countryid, 0) + 1
                city_dict[cityid] = city_dict.get(cityid, 0) + 1
                poi_ids.append(poiid)
                if poiid in self.poi_topic : 
                    for topic in self.poi_topic[poiid][0]: topic_ids.append(topic)
                    cate_ids.append(self.poi_topic[poiid][1])
            for poi_id in poi_ids: poi_dict[poi_id] = poi_dict.get(poi_id, 0) + 1
            for cate_id in cate_ids: cateid_dict[cate_id] = cateid_dict.get(cate_id,0) + 1
            for topic_id in topic_ids: topic_dict[topic_id] = topic_dict.get(topic_id,0) + 1

        city_detail = place_detail["city"]
        if len(city_detail) :
            for city_id in city_detail :
                city_dict[int(city_id)] = city_dict.get(int(city_id), 0) + 1
                if city_id in self.city_country : 
                    country_id = int(self.city_country[city_id])
                    country_dict[country_id] = country_dict.get(country_id, 0) + 1
 
        country_detail = place_detail["country"]
        for country_id in country_detail : 
            country_id = int(country_id)
            if country_id != 11 : country_dict[country_id] = country_dict.get(country_id, 0) + 1
        return {"country":country_dict, "city":city_dict,'poi':poi_dict, "topic_id": topic_dict, "cate_id": cateid_dict}

    #################################################################
    #功能:给行程助手打目的地标签                                    #
    #输入:plan_list - planid列表                                    #
    #输出:字典                                                      #
    #     {"country":{country_id:country_id在产品列表中出现的次数}, #
    #      "city":{city_id:city_id在产品列表中出现的次数}}          #
    #################################################################
    def plan_tag(self, plan_list) :
        if len(plan_list) == 0 : return {"country":{}, "city":{}}
        plan_ids_str = ",".join(map(str, plan_list))
        if len(self.city_country) == 0 : self.init_city_country()
        country_dict = {}
        city_dict = {}
        poi_dict = {}
        cateid_dict = {}
        topic_dict = {}
        self.plan_db.execute(horizonal_sql("select plan_id ,pid from plan_onedayevent_h where typeid = 3 and plan_id in (%s)" % (plan_ids_str,)))
        result_set = self.plan_db.fetchall()
        
        topic_ids = []
        cate_ids = []
        for result in result_set :
            pid = int(result['pid'])
            poi_dict[pid] = poi_dict.get(pid,0) + 1
            if pid in self.poi_topic : 
                for topic in self.poi_topic[pid][0]: topic_ids.append(topic)
                cate_ids.append(self.poi_topic[pid][1])
        for cate_id in cate_ids: cateid_dict[cate_id] = cateid_dict.get(cate_id,0) + 1
        for topic_id in topic_ids: topic_dict[topic_id] = topic_dict.get(topic_id,0) + 1

        self.plan_db.execute("select id, countryids, cityids from plan_route where id in (%s)" % plan_ids_str)
        result_set = self.plan_db.fetchall()
        for result in result_set :
            plan_id = result["id"]
            try : country_ids = map(int, result["countryids"].split(","))
            except Exception, e : country_ids = []
            for country_id in country_ids : 
                if country_id != 11 : country_dict[country_id] = country_dict.get(country_id, 0) + 1
            try : city_ids = map(int, result["cityids"].split(","))
            except  Exception, e : city_ids = []
            if len(self.taiwan_set & set(city_ids)) : country_dict[-11186] = country_dict.get(-11186, 0) + 1
            for city_id in city_ids : 
                city_dict[city_id] = city_dict.get(city_id, 0) + 1
                if city_id == 50 or city_id == 51 : country_dict[-city_id] = country_dict.get(-city_id, 0) + 1
        return {"country":country_dict, "city":city_dict,'poi':poi_dict, "topic_id": topic_dict, "cate_id": cateid_dict}

    ##################################################################
    #功能:给搜索打目的地标签                                         #
    #输入:search_detail {"wd_code":[搜索关键词ID], "wd":[搜索关键词]}#
    #输出:字典                                                     #
    #     {"country":{country_id:country_id在产品列表中出现的次数},  #
    #      "city":{city_id:city_id在产品列表中出现的次数}}           #
    ##################################################################
    def search_tag(self, search_detail) :
        if len(self.city_country) == 0 : self.city_country = self.init_city_country()
        country_dict = {}
        city_dict = {}
        words = []
        wd_code_list = search_detail.get("wd_code", [])
        for wd_code in wd_code_list :
            if not wd_code:
                continue
            self.qyer_db.execute("select name from qyer.search_keyword where id = %s", (wd_code,))
            result = self.qyer_db.fetchone()
            if not result : continue
            words.append(result["name"])
        wd_list = search_detail.get("wd", [])
        for wd in wd_list :
            segs = jieba.cut(wd)
            words.extend([strdecode(w).encode("utf8", "ignore") for w in segs if len(w) > 1])
        for word in words :
            if not word:
                continue
            self.qyer_db.execute("select id from category where (catename = %s or catename_en = %s) and parentid in (select id from category where parentid = 1) limit 1", (word, word))
            country_result = self.qyer_db.fetchone()
            if country_result and country_result["id"] != 11 : country_dict[country_result["id"]] = country_dict.get(country_result["id"], 0) + 1
            self.qyer_db.execute("select id, countryid from city where (cityname = '%s' or cityname like '%s州' or cityname like '%s省' or cityname like '%s县' or cityname like '%s市' or cityname_en = '%s') and status = 1" % (sql_transform(word), sql_transform(word), sql_transform(word), sql_transform(word), sql_transform(word), sql_transform(word)))
            city_result = self.qyer_db.fetchone()
            if city_result and city_result["id"] in self.city_country :
                cityid = city_result["id"]
                countryid = self.city_country[cityid]
                country_dict[countryid] = country_dict.get(countryid, 0) + 1
                city_dict[cityid] = city_dict.get(cityid, 0) + 1
        city_ids = search_detail.get("cityids", [])
        if len(city_ids) :
            self.qyer_db.execute("select id, countryid from city where id in (%s)" % ",".join(map(str, city_ids)))
            city_result_set = self.qyer_db.fetchall()
            for city_result in city_result_set :
                cityid = city_result["id"]
                if cityid not in self.city_country : continue
                countryid = self.city_country[cityid]
                country_dict[countryid] = country_dict.get(countryid, 0) + 1
                city_dict[cityid] = city_dict.get(cityid, 0) + 1
        country_ids = search_detail.get("countryids", [])
        for country_id in country_ids : 
            if country_id != 11 : country_dict[country_id] = country_dict.get(country_id, 0) + 1
        return {"country":country_dict, "city":city_dict}

    ##############################################################################
    #功能:给折扣打目的地标签                                                     #
    #输入:折扣id列表                                                             #
    #输出:字典                                                                   #
    #     {"country":{country_id:country_id在产品列表中出现的次数},              #
    #      "city":{city_id:city_id在产品列表中出现的次数},                       #
    #      "product_type":{product_type_id:产品类型在列表中出现的次数},          #
    #      "departure":{出发地:出发地在产品列表中出现的次数}}                    #
    ##############################################################################
    def lm_tag(self, lmid_list) :
        country_dict = {}
        city_dict = {}
        product_type_dict = {}
        departure_dict = {}
        poiid_dict = {}
        topic_dict = {}
        cateid_dict = {}
        for lmid in lmid_list :
            self.qyer_db.execute("select product_type, country, city, abroad, poi from qyer_lastminute.lastminute_main where id = %s", (lmid,))
            lm_res = self.qyer_db.fetchone()
            if lm_res :
                countryids = set([])
                country_names = lm_res['country'].split(",")
                poi_ids = []
                for p in lm_res['poi'].strip().split(","):
                    if p.isdigit(): poi_ids.append(int(p))
                city_names = lm_res['city'].split(",")
                product_type_dict[lm_res["product_type"]] = product_type_dict.get(lm_res["product_type"], 0) + 1
                departure_dict[lm_res["abroad"]] = departure_dict.get(lm_res["abroad"], 0) + 1
                if len(country_names) > 0 and len(country_names) <= 2 :
                    country_names_str = '","'.join(country_names)
                    if not country_names_str : continue
                    self.qyer_db.execute('select id from category where catename in ("%s") and parentid in (select id from category where parentid = 1)' % country_names_str)
                    country_result_set = self.qyer_db.fetchall()
                    countryids = set([country_result["id"] for country_result in country_result_set if country_result["id"] != 11])
                cate_ids = []
                topic_ids = []
                if len(poi_ids) > 0:
                    for poi in poi_ids:
                        poiid_dict[poi] = poiid_dict.get(poi,0) + 1
                        if poi in self.poi_topic : 
                            for topic in self.poi_topic[poi][0]: topic_ids.append(topic)
                            cate_ids.append(self.poi_topic[poi][1])
                for cate_id in cate_ids: cateid_dict[cate_id] = cateid_dict.get(cate_id,0) + 1
                for topic_id in topic_ids: topic_dict[topic_id] = topic_dict.get(topic_id,0) + 1

                if len(city_names) > 0 and len(city_names) <= 2 :
                    city_names_str = '","'.join(city_names)
                    if not city_names_str : continue
                    self.qyer_db.execute('select id, countryid from city where cityname in ("%s")' % city_names_str)
                    city_result_set = self.qyer_db.fetchall()
                    cityids = [city_result["id"] for city_result in city_result_set]
                    countryids = countryids | set([self.city_country[cityid] for cityid in cityids if cityid in self.city_country])
                    for cityid in cityids : city_dict[cityid] = city_dict.get(cityid, 0) + 1
                for countryid in countryids : country_dict[countryid] = country_dict.get(countryid, 0) + 1
        return {"country":country_dict, "city":city_dict, "product_type":product_type_dict, "departure":departure_dict,'poi':poiid_dict, "topic_id": topic_dict, "cate_id": cateid_dict}

    ##############################################################################
    #功能:给锦囊集合打目的地标签                                                 #
    #输入:锦囊id集合                                                             #
    #输出:字典                                                                   #
    #     {"country":{country_id:country_id在产品列表中出现的次数},              #
    #      "city":{city_id:city_id在产品列表中出现的次数}}                       #
    ##############################################################################
    def guide_tag(self, guide_set) :
        country_dict = {}
        city_dict = {}
        guide_ids = list(guide_set)
        if len(guide_ids) == 0 : return {"country":country_dict, "city":city_dict}
        guide_ids_str = ",".join(map(str, guide_ids))
        self.qyer_db.execute("select id, country, poi from qyer.jn_info where id in (%s)" % guide_ids_str)
        result_set = self.qyer_db.fetchall()
        for result in result_set :
            country_ids = result['country'].split(",")
            for country_id in country_ids :
                if country_id and country_id.isdigit() and country_id != '11' : country_dict[int(country_id)] = country_dict.get(int(country_id), 0) + 1
            try : city_ids = map(int, result['poi'].split(","))
            except : continue
            if len(self.taiwan_set & set(city_ids)) : country_dict[-11186] = country_dict.get(-11186, 0) + 1
            for city_id in city_ids :
                if city_id == 50 or city_id == 51 : country_dict[-city_id] = country_dict.get(-city_id, 0) + 1
                city_dict[city_id] = city_dict.get(city_id, 0) + 1
        return {"country":country_dict, "city":city_dict}

    ##############################################################################
    #功能:给微锦囊列表打目的地标签                                               #
    #输入:微锦囊id列表                                                           #
    #输出:字典                                                                   #
    #     {"country":{country_id:country_id在产品列表中出现的次数},              #
    #      "city":{city_id:city_id在产品列表中出现的次数}}                       #
    ##############################################################################
    def mguide_tag(self, mgid_list) :
        country_dict = {}
        city_dict = {}
        poiid_dict = {}
        topic_dict = {}
        cateid_dict = {}
        if len(mgid_list) == 0 : return {"country":{}, "city":{}}
        mgid_list_str = ",".join(map(str, mgid_list))
        self.qyer_db.execute("select belong_id from place_micro_guide_belong where belong_type = 4 and micro_guide_id in (%s)" % mgid_list_str)
        result_set = self.qyer_db.fetchall()
        cate_ids = []
        topic_ids = []
        for result in result_set:
            poiid = result['belong_id']
            poiid_dict[poiid] = poiid_dict.get(poiid,0) + 1
            if poiid in self.poi_topic : 
                for topic in self.poi_topic[poiid][0]: topic_ids.append(topic)
                cate_ids.append(self.poi_topic[poiid][1])
        for cate_id in cate_ids: cateid_dict[cate_id] = cateid_dict.get(cate_id,0) + 1
        for topic_id in topic_ids: topic_dict[topic_id] = topic_dict.get(topic_id,0) + 1

        self.qyer_db.execute("select belong_id from place_micro_guide_belong where belong_type = 2 and micro_guide_id in (%s)" % mgid_list_str)
        result_set = self.qyer_db.fetchall()
        for result in result_set : 
            if result["belong_id"] != 11 : country_dict[result["belong_id"]] = country_dict.get(result["belong_id"], 0) + 1
        self.qyer_db.execute("select belong_id from place_micro_guide_belong where belong_type = 3 and micro_guide_id in (%s)" % mgid_list_str)
        result_set = self.qyer_db.fetchall()
        cityids = [result["belong_id"] for result in result_set]
        if len(self.taiwan_set & set(cityids)) : country_dict[-11186] = country_dict.get(-11186, 0) + 1
        for cityid in cityids : 
            city_dict[cityid] = city_dict.get(cityid, 0) + 1
            if cityid == 50 or cityid == 51 : country_dict[-cityid] = country_dict.get(-cityid, 0) + 1
        return {"country":country_dict, "city":city_dict, 'poi':poiid_dict, "topic_id": topic_dict, "cate_id": cateid_dict}

    ##############################################################################
    #功能:给折扣列表打目的地标签                                                 #
    #输入:折扣列表                                                               #
    #输出:字典                                                                   #
    #     {"country":{country_id:country_id在产品列表中出现的次数},              #
    #      "city":{city_id:city_id在产品列表中出现的次数}}                       #
    ##############################################################################
    def lmlist_tag(self, lmlist_details) :
        country_dict = {}
        city_dict = {}
        country_ids = lmlist_details["country"]
        for country_id in country_ids : 
            if country_id != 11 : country_dict[country_id] = country_dict.get(country_id, 0) + 1
        city_ids = lmlist_details["city"]
        for city_id in city_ids : 
            city_dict[city_id] = city_dict.get(city_id, 0) + 1
            if city_id not in self.city_country : continue
            country_id = self.city_country[city_id]
            if country_id < 0 : country_dict[country_id] = country_dict.get(country_id, 0) + 1
        return {"country":country_dict, "city":city_dict}

    ##############################################################################
    #功能:给酒店列表打目的地标签                                                 #
    #输入:酒店id列表                                                             #
    #输出:字典                                                                   #
    #     {"country":{country_id:country_id在产品列表中出现的次数},              #
    #      "city":{city_id:city_id在产品列表中出现的次数}}                       #
    ##############################################################################
    def hotel_tag(self, hotel_list) :
        country_dict = {}
        city_dict = {}
        if len(hotel_list) == 0 : return {"country":country_dict, "city":city_dict}
        hotel_ids_str = ",".join(map(str, hotel_list))
        self.qyer_db.execute("select id, qyer_country_id, qyer_city_id from qyer.business_hotel where id in (%s)" % hotel_ids_str)
        result_set = self.qyer_db.fetchall()
        for result in result_set :
            if result["qyer_country_id"] : 
                if result["qyer_country_id"] != 11 : country_dict[result["qyer_country_id"]] = country_dict.get(result["qyer_country_id"], 0) + 1
            if result["qyer_city_id"] : 
                city_id = result["qyer_city_id"]
                city_dict[city_id] = city_dict.get(city_id, 0) + 1
                if city_id not in self.city_country : continue
                country_id = self.city_country[city_id]
                if country_id < 0 : country_dict[country_id] = country_dict.get(country_id, 0) + 1
        return {"country":country_dict, "city":city_dict}
        

    ##############################################################################
    #功能:给X列表打目的地标签                                                    #
    #输入:X id列表                                                               #
    #输出:字典                                                                   #
    #     {"country":{country_id:country_id在产品列表中出现的次数},              #
    #      "city":{city_id:city_id在产品列表中出现的次数}}                       #
    ##############################################################################
    def xlist_tag(self, hotel_details) :
        country_dict = {}
        city_dict = {}
        country_ids = hotel_details.get("country", [])
        for country_id in country_ids : 
            if country_id != 11 : country_dict[country_id] = country_dict.get(country_id, 0) + 1
        city_ids = hotel_details.get("city", [])
        for city_id in city_ids : 
            city_dict[city_id] = city_dict.get(city_id, 0) + 1
            if city_id in self.city_country :
                country_id = self.city_country[city_id]
                country_dict[country_id] = country_dict.get(country_id, 0) + 1
        return {"country":country_dict, "city":city_dict}

doraid_views={}
template = {"uids":set(), "v_tbbs":set(), "v_ask":set(), "v_plan":set(), "edit_plan":set(), "search":{"wd":[], "wd_code":[], "cityids":[], "countryids":[]}, "v_place":{"country":[], "city":[], "poi":[]}, "v_guide":set(), "v_lm":set(), "v_lmlist":{"country":[], "city":[]}, "d_guide":set(), "ask":[], "answer":set([]), "p_comment":{"country":[], "city":[], "poi":[]}, "coplay":[], "p_travel":[], "v_mguide":set([]), "v_hotel":[], "buy_lm":[], "collect_lm":[], "v_hotellist":{"country":[], "city":[]}, "book_hotel":[], "v_goto_hotel":[], "v_glist":{"country":[], "city":[]}}
specialType={"search":{"wd":types.ListType, "wd_code":types.ListType, "cityids":types.ListType, "countryids":types.ListType}, "v_place":{"country":"", "city":"", "poi":""}, "v_lmlist":{"country":"", "city":""}, "p_comment":{"country":"", "city":"", "poi":""}, "v_glist":{"country":"", "city":""}, "v_hotellist":{"country":"", "city":""}}

if __name__ == '__main__':
    pt = ProductTagger()
    for line in sys.stdin:
        rowSet = line.strip().split('\t')
        if (len(rowSet) < 4):
            continue        
        dora_id = rowSet[0]
        uids = rowSet[1]
        parsed_type = rowSet[2]
        #解析成str list
        parsed_content = eval(rowSet[3]) 
        if parsed_type == "search":
            continue
        tmp=copy.deepcopy(template[parsed_type])
        collect_type = type(template[parsed_type])
        #字典类型的,合并
        if collect_type is types.DictType :
            for elem in  parsed_content:
                #str解析成字典
                elem = eval(elem)
                for k,v in elem.iteritems():
                    if specialType[parsed_type][k] is types.ListType:
                        tmp[k].extend(v)
                    else:
                        tmp[k].append(v)
        elif collect_type is set:
            tmp.update(parsed_content)
            tmp=list(tmp)
        elif collect_type is list:
            tmp.extend(parsed_content)
        else:
            continue
        
        parsed_content=tmp
#         t1=datetime.datetime.now()
        parsed_tags = pt.product_tag(parsed_type, parsed_content)
#         t2=datetime.datetime.now()
#         print "%s==%s==%s" % (t2-t1,parsed_type,rowSet[3])
        print "%s\t%s\t%s\t%s\t%s" % (dora_id,uids,parsed_type,parsed_content,json.dumps(parsed_tags))
        
